import requests
import threading
from lxml import html

class Spider(threading.Thread):
    def __init__(self,title,url):
        threading.Thread.__init__(self)
        self.title = title
        self.url = url

    def run(self):
        Runing(self.title,self.url)



def Runing(title,url):
    for i in range(1,7):
        urls = "https://m.bqg99.cc/"+url+"_"+str(i)+".html"
        response = requests.get(url=urls,headers=headers).text
        resp = html.etree.HTML(response)
        content = resp.xpath('//div[@id="chaptercontent"]/text()')
        contents = ' '.join(content).strip()
        with open('D:/雪中悍刀行/{}.txt'.format(title),'a',encoding='utf-8') as f:
            f.write(contents+"\n")
            print(f'已经爬取:{title}')
        f.close()




if __name__ == '__main__':
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36 Edg/101.0.1210.32',
        'referer':'https://m.bqg99.cc/book/1036902218/'

    }
    url = "https://m.bqg99.cc/book/1036902218/all.html"

    response = requests.get(url=url,headers=headers).text
    # print(response)
    resp = html.etree.HTML(response)
    titles = resp.xpath('//div[@class="book_last"]/dl//dd/a/text()')
    # 获取文章的所有标题
    title = [i for i in titles if i !='↓↓↓ 直达页面底部']
    # 获取文章的所有url
    title_urls = resp.xpath('//div[@class="book_last"]/dl//dd/a/@href')
    title_urlss = [i for i in title_urls if i != '#footer']
    title_url = []
    for i in title_urlss:
        # title_url.append(i.strip('.html'))
        resp = i.strip('.html')
        title_url.append(resp)
    # print(title_url)

    for i in range(0,len(title_url)):
        threading.Semaphore(100)
        Travel = Spider(title[i],title_url[i])
        Travel.start()
        # Travel.join()